using Statistics
# A list of stopwords to remove from the text
stopwords=["as", "a", "able", "about", "above", "according", "accordingly", "across", "actually",
    "after", "afterwards", "again", "against", "ain", "t", "all", "allow", "allows", "almost", "alone",
    "along", "already", "also", "although", "always", "am", "among", "amongst", "an", "and", "another",
    "any", "anybody", "anyhow", "anyone", "anything", "anyway", "anyways", "anywhere", "apart", "appear",
    "appreciate", "appropriate", "are", "aren", "t", "around", "as", "aside", "ask", "asking", "associated",
    "at", "available", "away", "awfully", "be", "became", "because", "become", "becomes", "becoming",
    "been", "before", "beforehand", "behind", "being", "believe", "below", "beside", "besides", "best",
    "better", "between", "beyond", "both", "brief", "but", "by", "c", "mon", "c", "s", "came", "can",
    "can", "t", "cannot", "cant", "cause", "causes", "certain", "certainly", "changes", "clearly", "co",
    "com", "come", "comes", "concerning", "consequently", "consider", "considering", "contain", "containing",
    "contains", "corresponding", "could", "couldn", "t", "course", "currently", "definitely", "described",
    "despite", "did", "didn", "t", "different", "do", "does", "doesn", "t", "doing", "don", "t", "done", "down",
    "downwards", "during", "each", "edu", "eg", "eight", "either", "else", "elsewhere", "enough",
    "entirely", "especially", "et", "etc", "even", "ever", "every", "everybody", "everyone", "everything",
    "everywhere", "ex", "exactly", "example", "except", "far", "few", "fifth", "first", "five", "followed",
    "following", "follows", "for", "former", "formerly", "forth", "four", "from", "further", "furthermore",
    "get", "gets", "getting", "given", "gives", "go", "goes", "going", "gone", "got", "gotten", "greetings",
    "had", "hadn", "t", "happens", "hardly", "has", "hasn", "t", "have", "haven", "t", "having", "he", "he", "s",
    "hello", "help", "hence", "her", "here", "here", "s", "hereafter", "hereby", "herein", "hereupon", "hers",
    "herself", "hi", "him", "himself", "his", "hither", "hopefully", "how", "howbeit", "however", "i", "d",
    "i", "ll", "i", "m", "i", "ve", "ie", "if", "ignored", "immediate", "in", "inasmuch", "inc", "indeed", "indicate",
    "indicated", "indicates", "inner", "insofar", "instead", "into", "inward", "is", "isn", "t", "it", "it", "d",
    "it", "ll", "it", "s", "its", "itself", "just", "keep", "keeps", "kept", "know", "knows", "known", "last",
    "lately", "later", "latter", "latterly", "least", "less", "lest", "let", "let", "s", "like", "liked",
    "likely", "little", "look", "looking", "looks", "ltd", "made", "mainly", "many", "may", "maybe", "me", "mean",
    "meanwhile", "merely", "might", "more", "moreover", "most", "mostly", "much", "must", "my", "myself",
    "name", "namely", "nd", "near", "nearly", "necessary", "need", "needs", "neither", "never", "nevertheless",
    "new", "next", "nine", "no", "nobody", "non", "none", "noone", "nor", "normally", "not", "nothing",
    "novel", "now", "nowhere", "obviously", "of", "off", "often", "oh", "ok", "okay", "old", "on", "once",
    "one", "ones", "only", "onto", "or", "other", "others", "otherwise", "ought", "our", "ours", "ourselves",
    "out", "outside", "over", "overall", "own", "particular", "particularly", "per", "perhaps", "placed", "please",
    "plus", "possible", "presumably", "probably", "provides", "que", "quite", "qv", "rather", "rd", "re", "really",
    "reasonably", "regarding", "regardless", "regards", "relatively", "respectively", "right", "said", "same", "saw",
    "say", "saying", "says", "second", "secondly", "see", "seeing", "seem", "seemed", "seeming", "seems", "seen",
    "self", "selves", "sensible", "sent", "serious", "seriously", "seven", "several", "shall", "she", "should",
    "shouldn", "t", "since", "six", "so", "some", "somebody", "somehow", "someone", "something", "sometime",
    "sometimes", "somewhat", "somewhere", "soon", "sorry", "specified", "specify", "specifying", "still", "sub",
    "such", "sup", "sure", "t", "s", "take", "taken", "tell", "tends", "th", "than", "thank", "thanks", "thanx",
    "that", "that", "s", "thats", "the", "their", "theirs", "them", "themselves", "then", "thence", "there",
    "there", "s", "thereafter", "thereby", "therefore", "therein", "theres", "thereupon", "these", "they", "they", "d",
    "they", "ll", "they", "re", "they", "ve", "thing", "things", "think", "third", "this", "thorough", "thoroughly", "those", "though",
    "three", "through", "throughout", "thru", "thus", "to", "together", "too", "took", "toward", "towards",
    "tried", "tries", "truly", "try", "trying", "twice", "two", "un", "under", "unfortunately", "unless",
    "unlikely", "until", "unto", "up", "upon", "us", "use", "used", "useful", "uses", "using", "usually",
    "value", "various", "very", "via", "viz", "vs", "want", "wants", "was", "wasn", "t", "way", "we", "we", "d",
    "we", "ll", "we", "re", "we", "ve", "welcome", "well", "went", "were", "weren", "t", "what", "what", "s", "whatever",
    "when", "whence", "whenever", "where", "where", "s", "whereafter", "whereas", "whereby", "wherein", "whereupon",
    "wherever", "whether", "which", "while", "whither", "who", "who", "s", "whoever", "whole", "whom", "whose",
    "why", "will", "willing", "wish", "with", "within", "without", "won", "t", "wonder", "would", "would",
    "wouldn", "t", "yes", "yet", "you", "you", "d", "you", "ll", "you", "re", "you", "ve", "your", "yours", "yourself", "yourselves", "zero"];

# cleantext grabs the full text without whitespace, punctuation, or stopwords
function cleantext(text)
    idx = findall(occursin.("***",text))
    text = text[idx[1]+1:idx[2]-1]
    text = replace(join(text[text.!=""]," "),r"\p{P}|[0-9]" => " ")
    text = lowercase(replace(text, r"\s{2,}" => " "))

    for w in stopwords
        text = replace(text," "*w*" " => " ");
    end

    return text
end

# countwords constructs a dictionary of the words that appear in the text as the keys and their counts as the values
function countwords(text)
    textSep = split(text," ")
    wordcounts = Dict{String,Float64}()
    for w in textSep
        wordcounts[w] = get(wordcounts, w, 0.0) + 1.0
    end
    return wordcounts
end

function getcounts(fname)
    f = open(fname)
    text=cleantext(readlines(f))
    return countwords(text)
end

function getnormcount(counts,overallcounts)

    # Calculate normalized score
    thresh = quantile(collect(values(counts)), .98)
    counts = filter!(word-> word.second > thresh, counts)
    normscore = merge(/,counts,filter(word->haskey(counts,word.first), overallcounts))

    # Rank the resulting scores
    rankedwords = sort(collect(normscore), by=x->x[2], rev=true)
    return rankedwords
end
